package com.lewis.spider;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.lewis.spider.pojo.Brand;
import com.lewis.spider.pojo.Cat;
import com.lewis.spider.pojo.CategoryPair;
import com.lewis.spider.pojo.Cats;
import com.lewis.spider.pojo.Item;
import com.thoughtworks.xstream.XStream;
import com.thoughtworks.xstream.annotations.Annotations;
import com.thoughtworks.xstream.io.xml.DomDriver;

public class SpiderStart {
	
	private static String url = "http://s.taobao.com/search?initiative_id=staobaoz_20130628&tab=all&q=%C3%C0%B9%FA+%B4%FA%B9%BA&source=haiwaigou&style=list&cps=yes&stats_click=search_radio_all%3A1&promote=0";
	
	public static void main(String[] args) {
		
		CategoryPair c1 = new CategoryPair();
		c1.setCode("1801");
		c1.setName("美容护肤");
		c1.setQuantity("7.4万");
		CategoryPair c2 = new CategoryPair();
		c2.setCode("50102996");
		c2.setName("女装");
		c2.setQuantity("27.5万");
		CategoryPair c3 = new CategoryPair();
		c3.setCode("50006842");
		c3.setName("女包");
		c3.setQuantity("17.9万");
		List<CategoryPair> cList = new ArrayList<CategoryPair>();
		cList.add(c1);
		cList.add(c2);
		cList.add(c3);
		
		HttpClient httpClient = new DefaultHttpClient();
		Cats cats = new Cats();
		
		for(CategoryPair c:cList ){
			StringBuilder sb = new StringBuilder(url);
			sb.append("&cat="+c.getCode());
			
	        Cat cat = new Cat();
	        cats.getCats().add(cat);
	        cat.setName(c.getName());
	        cat.setQuantity(c.getQuantity());
			HttpUriRequest httpUriRequest = new HttpGet(sb.toString());
			
			HttpResponse httpResponse = null;
			try {
				httpResponse = httpClient.execute(httpUriRequest);
			} catch (ClientProtocolException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			
			HttpEntity entity = httpResponse.getEntity();
			
			try {
				String html = new String(EntityUtils.toString(entity).getBytes("gbk"));
//		        File file = new File("/Users/henrylian/Desktop/Test1.html");
//		        FileWriter fwriter = new FileWriter(file);
//		        fwriter.write(html);
//		        fwriter.close();
		        Document doc = Jsoup.parse(html);
		        Element content = doc.getElementById("page");
		        Element form = content.getElementsByTag("form").get(3);
		        Element ul = form.getElementsByTag("ul").get(0);
		        Elements list = ul.getElementsByTag("li");
		        

		        for (Element ele: list){
		        	Element ahref = ele.getElementsByTag("a").get(0);
		        	System.out.println("title: " + ahref.text() + " href: " + ahref.attr("href"));
		        	Brand brand = new Brand();
		        	brand.setName(ahref.text());
		        	StringBuilder url = new StringBuilder("http://s.taobao.com");
		        	url.append(ahref.attr("href"));
		        	url.append("&sort=sale-desc");
		        	HttpUriRequest urlRequest = new HttpGet(url.toString());
		        	HttpResponse urlResponse = httpClient.execute(urlRequest);
		        	String urlHtml = new String(EntityUtils.toString(urlResponse.getEntity()).getBytes("gbk"));
		        	urlHtml = urlHtml.replaceAll("<textarea class=\"ks-datalazyload-custom\">", "");
//			        File file = new File("/Users/henrylian/Desktop/Test.html");
//			        FileWriter fwriter = new FileWriter(file);
//			        fwriter.write(urlHtml);
//			        fwriter.close();
		        	Document urldoc = Jsoup.parse(urlHtml);
		        	Element listview = urldoc.getElementsByClass("list-view").get(0);
		        	Elements divList = listview.getElementsByClass("item");
		        	for (Element e:divList){
		        		Item item = new Item();
		        		Element img = e.getElementsByTag("img").get(0);
		        		item.setPicUrl(img.attr("src").toString());
		        		Element bigImg = e.getElementsByClass("pic-box-big").get(0).getElementsByTag("img").get(0);
		        		item.setBigPicUrl(bigImg.attr("src").toString());
		        		Element a = e.getElementsByClass("summary").get(0).getElementsByTag("a").get(0);
		        		item.setTitle(a.attr("title").toString());
		        		item.setItemUrl(a.attr("href").toString());
		        		item.setPrice(e.getElementsByClass("price").get(0).text());
		        		item.setShippingPrice(e.getElementsByClass("shipping").get(0).text());
		        		item.setDealCount(e.getElementsByClass("dealing").get(0).text());
		        		Elements count = e.getElementsByClass("count");
		        		if (count.size() != 0){
		        			item.setCommentsCount(e.getElementsByClass("count").get(0).getElementsByTag("a").get(0).text());
		        		}
		        		item.setSellerName(e.getElementsByClass("seller").get(0).getElementsByTag("a").get(0).text());
		        		item.setSellerShop(e.getElementsByClass("seller").get(0).getElementsByTag("a").get(0).attr("href").toString());
		        		item.setSellerLocation(e.getElementsByClass("loc").get(0).text());
		        		System.out.println(item.toString());
		        		brand.getItems().add(item);
		        	}
		        	cat.getBrands().add(brand);
//		        	break;
		        }
			} catch (IllegalStateException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		

		
		//to xml
        XStream xstream = new XStream(new DomDriver());
        Annotations.configureAliases(xstream, Cats.class);
        Annotations.configureAliases(xstream, Cat.class);
        Annotations.configureAliases(xstream, Brand.class);
        Annotations.configureAliases(xstream, Item.class);
        String obj_xml=xstream.toXML(cats);
        System.out.println(obj_xml);
        
        
        
        try {
        	File file = new File("/Users/henrylian/Desktop/data.xml");
        	FileWriter fwriter = new FileWriter(file);
			fwriter.write(obj_xml);
			fwriter.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
        
		
	}

}
